<?php
// Report simple running errors
error_reporting(E_ERROR | E_WARNING | E_PARSE);
require('D:/Projects/gagcrawler/includes/functions.php');
#require('/home/wap247/public_html/9gag/includes/functions.php');
$latestUrl = 'http://www.haivl.com/new/';

function getHaivlAllPost($latestUrl){
	// create a new cURL resource
	$ch = curl_init();

	$header[0] = "Accept: text/xml,application/xml,application/xhtml+xml,";
	$header[0] .= "text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
	$header[] = "Cache-Control: max-age=0";
	$header[] = "Connection: keep-alive";
	$header[] = "Keep-Alive: 300";
	$header[] = "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7";
	$header[] = "Accept-Language: en-us,en;q=0.5";
	$header[] = "Pragma: "; // browsers keep this blank.
	// set URL and other appropriate options
	curl_setopt($ch, CURLOPT_URL, $latestUrl);
	curl_setopt($ch, CURLOPT_HEADER, 0);
	curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
	//curl_setopt($ch, CURLOPT_PROXY, '210.245.0.24:80');
	curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

	// grab URL and pass it to the browser
	$responseContent = curl_exec($ch);
	if(preg_match('/<div id="leftColumn">(.*?)<div id="rightColumn">/is', $responseContent, $mainContent)){
		$arrItems = explode('<div class="photoListItem"', $mainContent[1]);
		if(!empty($arrItems)){
			$postTitle = '';
			$postImage = '';
			$type = 'photo';
			foreach($arrItems as $item){
				//if(strstr($item, 'data-id') && !strstr($item, ' class="videoIndicator"')){
				if(strstr($item, 'data-id')){
					$htmlItem = strip_tags('<div '.$item, '<a><img>');
					$htmlItem = str_replace('-400.', '-650.', $htmlItem);
					if(preg_match('/<img src="(.*?)"/is', $htmlItem, $imgSrc)){
						$postImage = $imgSrc[1];
						if(strstr($item, 'videoIndicator')){
							$type = 'video';
							$videoCode = $postImage;
						}
						else{
							$type = 'photo';
						}
					}
					else{
						continue;
					}
					if(preg_match('/<a.*?href="(.*?)".*?>(.*?)<\/a>/i', $htmlItem, $postInfo)){
						$postTitle = strip_tags($postInfo[2]);
						$postUrl = strip_tags($postInfo[1]);
					}
					else{
						continue;
					}
					$unique = md5($postImage);
					echo "\tcrawling [".$postUrl."] .. ";
					#echo $postTitle.'<em>'.$postImage.'</em><hr>';
					$postTitle = mysql_real_escape_string($postTitle);
					$postUrl = mysql_real_escape_string($postUrl);
					//$postImage = mysql_real_escape_string($postImage);

					$fileInfo = imageDownloader($postImage);
					if(!$fileInfo) continue;
					$postImage = $fileInfo['name'];

					$sqlInsert = "INSERT INTO data(original_url, post_title, post_image, post_unique, crawl_date, file_checksum,item_type,source_page) VALUES('".$postUrl."', '$postTitle', '$postImage', '$unique', NOW(), '".$fileInfo['checksum']."', '$type', 'haivl.com')";
					#$sqlInsert = mysql_real_escape_string($sqlInsert);

					if(mysql_query($sqlInsert)){
						echo "done\n";
					}
					else{
						echo "failed -> \n".mysql_error()."\n\n";
						die("\n-------BREAK-------\n");
					}
				}
			}
		}
	}

	// close cURL resource, and free up system resources
	curl_close($ch);
}
require('D:/Projects/gagcrawler/dbconnect.php');
#require('/home/wap247/public_html/9gag/dbconnect.php');
#echo '<ol>';
$numPage = 15;
for($i = 1; $i <= $numPage; $i++){
	#echo '<li>';
	echo "----------------------\nCrawling page $i/$numPage:\n";
	getHaivlAllPost($latestUrl.$i);
	sleep(5);
	#echo '</li>';
}
echo "\n----------------------[DONE]----------------------\n";
#echo '<ol>';
